import java.net.*;
import java.io.*;
import java.util.*;
import java.util.regex.*;

import javax.swing.text.*;
import javax.swing.text.html.*;
import javax.swing.text.html.parser.*;


/**
 * 
 * Handles all functions to do with extracting article text from a page
 * 
 * @author tomalteman
 *
 */
public class pageParser {
	
	private String text;
	private String desc;
	private String temp;
	
	private int opentag;
	private int start;
	private int end;
	private int isLink; 
	
	private int debug =0;
	
	//It 2 flags
	private int depth;
	private String space = "  ";
	private String tag;
	private ArrayList<pageLine> list;
	
	//String for only acquiring p text
	private String ptext;
	
	//Parsing from tables
	private ArrayList<pageLine> doc;
	
	private String pattern;
	
	boolean style;
	
	
	public pageParser()
	{
		
	}
	
    public static void main(String[] args)
    {
    	pageParser p = new pageParser();
    	String t = p.getText(
    			"http://www.ercrugby.com/eng/12_12515.php", 
    			"Leicester's acting head coach Richard Cockerill described Julien Dupy's last-minute try as 'world class' after the Frenchman sent the Tigers into yet another Heineken Cup semi-final. Dupy broke Bath hearts with a 20-metre break in the final play of the"
    	);
    	System.out.println(t);

    	
    	
    	//System.out.println(p.getLink("http://news.google.com/news/url?sa=t&ct2=us%2F0_0_s_1_0_t&usg=AFQjCNHuKPMrjTT7x5rNof5_pcpTGaPv-g&sig2=nDE2o99Y8KD_JsKYR_Fo5w&cid=0&ei=shyxSdipLpOi0QXZ4OjUAQ&url=http%3A%2F%2Fwww.thisisbristol.co.uk%2Fhomepage%2FBristol-pub-attack-murder-trial-halted%2Farticle-751798-detail%2Farticle.html"));
    }
    
    private void reset()
    {
    	text = "";
		//System.out.println("DESC IS ||" +desc+"||");
		opentag = 0;
		start = 0;
		end = 0;
		depth = 0;
		isLink = 0;
		ptext = "";
		pattern = "";
		style = false;
		
		list = new ArrayList<pageLine>();
		doc = new ArrayList<pageLine>();
    }
    	
    
    public String getGuardian(String description)
    {
    	desc = description.trim();
    	
		reset();
		
        Reader r;
        
    	try {
	    	StringBuffer StringBuffer1 = new StringBuffer(description);
	        ByteArrayInputStream content = new ByteArrayInputStream(StringBuffer1.toString().getBytes("UTF-8"));
	        
	        r = new InputStreamReader((InputStream) content);
	        
	        HTMLEditorKit.Parser parser;
            parser = new ParserDelegator();
            pParser lp = new pParser();
            parser.parse(r, lp, true);
	        
	        
	        
    	} catch (Exception e) {
    		System.err.println("Error: " + e);
    	}
        
        return ptext;
    }
    
	/**
	 * 
	 * Given a link and the articles description, it will find as much of the full article text as possible
	 * 
	 * @param link
	 * @param description
	 * @return
	 */
	public String getText(String link, String description)
	{
		desc = description.trim();

		reset();
		
    	URL u;
        Reader r;
            	
        try {
        	u = new URL(link);
            Object content = u.getContent();
            r = new InputStreamReader((InputStream)content);

            HTMLEditorKit.Parser parser;
            parser = new ParserDelegator();
            pParser lp = new pParser();
            parser.parse(r, lp, true);
            
            //Comment out if not textparser
            //Print out list
            if(debug==1)
            {
	            for(int i=0;i<list.size();i++)
	            {
	            	list.get(i).print();
	            }
            }
            //System.out.println("DESC IS ||" +desc+"||");
            
            Pattern pat = patternFind(getSubstring(description));
            
            pText(pat);
            if(text.length() < description.length()) 
            	{
            		if(debug==2) System.out.println("Do search doc");
            		text = "";
            		searchDoc(pat);
            	}
            if(debug==2) System.out.println("PATTERN IS "+pat);
            
            //If still not found then shorten description and try again
            tryAgain(link,description);
            
        } catch (PatternSyntaxException pse) {
        	tryAgain(link,description);
        } catch (IOException ie) {
        	//Return this string if cannot connect
        	System.err.println("Connection Error: " + ie);
            text = "connectionerror";
        } catch (Exception e) {
            System.err.println("Error: " + e);
        }
		
		return text;
	}
	
	private void tryAgain(String link, String description)
	{
		if(text.length() < description.length())
        {
        	int end = description.indexOf(pattern);
        	
        	if(debug==3) System.out.println("FIND "+pattern+ " IN "+description);
        	
        	//Make sure string is long enough to be viable
        	if(end>5) getText(link,description.substring(0, end));
        	else if(debug==3) System.err.println("Too short: "+description.substring(0, end));
        }
	}
	
	/**
	 * Gets description string and returns a sub string comprised of no more than one sentence
	 * @param s
	 * @return
	 */
	private String getSubstring(String s)
	{
		String d = s.substring(desc.length()-20);
		int half = d.length() / 2;
		
		if(debug==1) System.out.println("SUBSTRING IS  "+d+" HALF IS "+half);
		
		//Go through array and check whether any fullstops
		//This indicates end of sentence.
		//Check whether half way through string - if so use first half
		for(int i=0;i<d.length();i++)
		{
			if(d.charAt(i) == '.')
			{
				if(i>half) return d.substring(0,i);
				else return d.substring(i+1);
			}
		}
		
		return d;
	}
	
	/**
	 * Removes ascii values not readable
	 * 
	 * @param s
	 * @return
	 */
	private String cleanText(String s)
	{
		char[] data = s.toCharArray();
		
		for(int i=0;i<data.length;i++) if(data[i] > 122) data[i] = 32;
		
		return new String(data);
	}
	
	/**
	 * Checks a link for whether it should be included
	 * Bad words are ones indiciating links not relevant to the story
	 * 
	 * @param data
	 * @return
	 */
	private String cleanLink(char[] data)
	{
		String s = new String(data);
		String t = s.toLowerCase();
		String[] bad = {"say","comment","click","stor"};
		
		for(int i=0;i<bad.length;i++) if(t.contains(bad[i])) return "";
		
		return (s);
	}
	
	/**
	 * Ensures search pattern is not found in wrong place
	 * @param s
	 * @return
	 */
	private boolean validTag(String s)
	{
		if(s.equals("title")) return false;
		else if(s.equals("h1")) return false;
		
		return true;
	}
	
	/**
	 * Test regular expression patterns
	 */
	private Pattern patternFind(String desc)
	{
		desc = desc.trim();
		pattern = desc;
		//System.out.println("DESC IS ||" +desc+"||");
		char[] data = new char[(desc.length() * 3)+2];
		char[] test = desc.toCharArray();
		int k=0;
		
		
		data[0] = '.';
		data[1] = '*';
		for(int i=2;i<data.length;i=i+3)
		{
			data[i] = test[k];
			data[i+1] = '.';
			data[i+2] = '?';
			k++;
		}
		
		if(debug==1) System.out.println(new String(data));
		
		return Pattern.compile(new String(data));
	}
	
	private void searchDoc(Pattern p)
	{
		pageLine pl;
		Matcher m;
		String tag= "";
		int i=0;
		int d=0;
		boolean read=false;
		boolean p_tag =false;
		style = false;
		
		if(debug==3) System.out.println("PATTERN IS "+p);
		
		//Find text
		while(i<doc.size())
		{
			pl = doc.get(i);
			
			if(debug==3)
			{
				if(pl.getClose()==1) System.out.print("/");
				System.out.print(i+" "+pl.getTag()+" "+pl.getDepth()+": "+pl.getData());
				System.out.println();
			}
			
			m = p.matcher(pl.getData());
			if(m.find() && validTag(pl.getTag()))
				{
					if(debug==3) System.out.println("FOUND AT "+ pl.getDepth() + " AND LINE "+pl.getData());
					d = pl.getDepth();
					break;
				}
			//if(debug==3) System.out.println("PATTERN IS "+p);
			i++;
		}
		
		//Find open tag at same depth - only if 
		while((i>-1) && (d>0))
		{
			pl = doc.get(i);
			
			//Only check open tags and store whichever is open for text
			if((pl.getClose()==2) && (pl.getDepth() == d))
			{
				tag = pl.getTag();
				break;
			}
			
			i--;
		}
		
		//Read in any text between tags 'tag' at depth 'd'
		while(i<doc.size())
		{
			pl = doc.get(i);
			
			//If open tag then start reading
			if((pl.getClose()==2) && (pl.getDepth()==d) && (pl.getTag().equals(tag))) read = true;
			
			//If style tag then dont store
			if((pl.getClose()==2) && (pl.getDepth()==d) && (pl.getTag().equals("style"))) 
				{
					style = true;
				}
			
			//If p tag embedded within read tag then read text in there as well
			if((pl.getClose()==2) && (pl.getDepth()>d) && read && (pl.getTag().equals("p"))) p_tag=true;
			
			//If text and read open then store data
			if((pl.getClose()==0)&&(pl.getDepth()==d)&&read&&!style) text = text + " " + pl.getData();
			
			//If p tag open then read in p text
			if((pl.getClose()==0)&&p_tag) text = text + " " + pl.getData();
			
			//Close p tag
			if((pl.getClose()==1) && (pl.getDepth()>d) && read && (pl.getTag().equals("p"))) p_tag=false;
			
			//If closing tag then stop reading
			if((pl.getClose()==1) && (pl.getDepth()==d) && (pl.getTag().equals(tag))) style = false;
			
			//If closing tag then stop reading
			if((pl.getClose()==1) && (pl.getDepth()==d) && (pl.getTag().equals(tag))) read = false;
			
			i++;
		}
	}
	
	/**
	 * Get article text from arraylist given pattern
	 */
	private void pText(Pattern p)
	{
		text = "";
		int pDepth = -1;
		int endLine = -1;
		int foundline = -1;
		int startline = 0;
		int i=0;
		Matcher m;
		pageLine pl;
		
		//Go through list until pattern is found
		while((i < list.size()) && (endLine < 0))
		{
			pl = list.get(i);
			
			if((pDepth < 0) && (pl.getClose() == 0) && (pl.getData().length() > 1))
			{
				
				if(debug==1) System.out.println("Looking for data "+pl.getData());
				m = p.matcher(pl.getData());
				if(m.find())
					{
						pDepth = pl.getDepth();
						foundline = i;
						if(debug==1) System.out.println("FOUND AT "+ pDepth + " AND LINE "+i);
					}
			}
			else if((pDepth > -1) && (pl.getClose() == 1) && (pl.getDepth() == pDepth-1) && (pl.getTag().equals("div")))
			{
				if(debug==1) 
					{
						System.out.println("ENDLINE FOUND AT "+ (pDepth-1) + " AND LINE "+i);
						System.out.println("LINE BEFORE IS "+list.get(i-1).getData());
					}
				endLine = i;
			}
			
			i++;
		}
		
		//If not foudn return
		if(foundline==-1) return ;
		
		i=foundline;
		//Look for encompassing DIV before foundline
		while(i>0)
		{
			pl = list.get(i);
			
			if((pl.getClose()==2) && (pl.getDepth() < pDepth))
			{
				startline = i;
				break;
			}
			
			i--;
		}
		
		//If endLine not found then simply get all p in same identation
		if(endLine < 0) endLine = list.size();
		
		//Once pattern found get text from pDepth until endLine
		for(i=startline;i<endLine;i++)
		{
			pl = list.get(i);
			if((pl.getClose()==0) && (pl.getDepth()==pDepth) && pl.getTag().equals("p")) text = text + pl.getData();
		}
		
		
		if(debug==1) System.out.println(text);
	}
	
	/**
	 * Insert regular expression to ensure acronyms are detected in article
	 * @param description
	 */
	private void setDesc(String description)
	{
		char[] letters = description.toCharArray();
		int k=0;
		
		//Create space for regex
		int size = letters.length * 5;
		char[] reg = new char[size];
		
		for(int i=0;i<reg.length;i=i+5)
		{
			reg[i] = letters[k];
			k++;
			reg[i+1]= '\\';
			reg[i+2]= '\\';
			reg[i+3]= '.';
			reg[i+4]= '*';
		}
		
		//System.out.println(new String(reg));
	}


	
	/**
	 * Articles parser if text is within p tags
	 * 
	 * iteration 3
	 * 
	 * @author tomalteman
	 *
	 */
	class pParser extends HTMLEditorKit.ParserCallback
	{
		public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) {
			if(t.equals(HTML.Tag.P))
				{
					start = 1;
					text = "";
				}
			else if(t.equals(HTML.Tag.A))
			{
				//Dont write link text
				isLink = 1;
			}
			
			if(t.equals(HTML.Tag.STYLE)) 
				{
					style = true;
					//System.out.println("STYLE");
				}
			
			//Set tag name
			tag = t.toString();
			
			//Avoid reading invalid text 
			//if(t.equals(HTML.Tag.STYLE)) valid = false;
			//else valid = true;

			//Open tag means add to depth
			depth++;
			
			
			if(debug==4) System.out.println(tag + depth);
			
			//Add to doc and list
			doc.add(new pageLine(tag,2,depth));
			list.add(new pageLine(tag,2,depth));
			
			if(debug==1)
			{
				System.out.print(t.toString()+ " ");
				System.out.print(depth);
				System.out.print(" "+a);
				System.out.println();
			}

			
		}
		
		public void handleText(char[] data,int pos)
        {
			if(debug==2) System.out.println(depth + new String(data));
			
			//Add to doc
			if(!style) doc.add(new pageLine(tag,0,depth,cleanText(new String(data))));
			
			if(start==1)
				{
					if(isLink==0) text = text + " " + new String(data);
					else 
						{
							text = text + cleanLink(data);
						}
					
					//Test by printing depth and data
					if(debug==1)
					{
						System.out.println(new String(data));
					}
				}
			
			//Test by printing depth and data
			if(debug==1)
			{
				if(tag.equals("div")) System.out.println(depth + " " + new String(data));
			}
        }
		
		public void handleEndTag(HTML.Tag t, int pos) {
			if(debug==2) System.out.println("/"+t.toString()+depth);
			if(t.equals(HTML.Tag.P))
				{
					start = 0;
					
					//Write to arraylist
					list.add(new pageLine("p",0,depth,cleanText(text)));
					
					//Add to ptext String
					ptext= ptext + cleanText(text);
				}
			else if(t.equals(HTML.Tag.A))
			{
				//Dont write link text
				isLink = 0;
			}
			else if(t.equals(HTML.Tag.STYLE)) style = false;
			//else if(t.equals(HTML.Tag.STYLE)) valid = true;
			
			tag = t.toString();
			
			//Add to doc
			doc.add(new pageLine(tag,1,depth));
			
			//Write end tag to list only if its an end div
			if(t.equals(HTML.Tag.DIV)) list.add(new pageLine(tag,depth));
			
			if(debug==1)
			{
				System.out.print(t+ " /");
				System.out.print(depth);
				System.out.println();
			}
			depth--;
		}
		
	}
	
	
	/**
	 * Articles parser
	 * 
	 * iteration 2
	 * 
	 * @author tomalteman
	 *
	 */
	class textParser extends HTMLEditorKit.ParserCallback
	{
		public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) {
			if(t.equals(HTML.Tag.P)) start = 1;

			//Open tag means add to depth
			depth++;
			
			//Set tag name
			tag = t.toString();
			
			if(debug==1)
			{
				System.out.print(t.toString()+ " ");
				System.out.print(depth);
				System.out.println();
			}

			
		}
		
		public void handleText(char[] data,int pos)
        {
			if(start==1)
				{
					text = text + new String(data);
					
					//Write to page list
					list.add(new pageLine(tag,0,depth,new String(data)));
					
					//Test by printing depth and data
					if(debug==1)
					{
						System.out.println(new String(data));
					}
				}
			
			
        }
		
		public void handleEndTag(HTML.Tag t, int pos) {
			if(t.equals(HTML.Tag.P)) start = 0;
			tag = t.toString();
			
			//Write end tag to list
			list.add(new pageLine(tag,depth));
			
			if(debug==1)
			{
				System.out.print(t+ " /");
				System.out.print(depth);
				System.out.println();
			}
			depth--;
		}
		
	}
	
	/**
	 * 
	 * Article page parser
	 * iteration 1
	 * 
	 * @author tomalteman
	 *
	 */
    class artParser extends HTMLEditorKit.ParserCallback
    {
    	/**
    	 * Format test string such that:
    	 * , removed from long number eg 20000 = 20,000
    	 * @param data
    	 * @return
    	 */
    	private String formatData(char[] data)
    	{
    		String s,t;
    		t= new String(data);
    	
    		
    		for(int i=1;i<data.length-1;i++)
    		{
    			if(Character.isDigit(data[i-1]) && (data[i] == ',') && Character.isDigit(data[i+1]))
    			{
    				s = t.substring(0,i) + t.substring(i+1,t.length());
    				t = s;
    			}
    		}
    		
    		return t;
    	}

        public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) {
            //If started then flag open tags
        	if(start==1) opentag =1;
        }
        
        public void handleText(char[] data,int pos)
        {
        	//System.out.println("Falg is "+end);
        	
        	//Unless checking for end
        	if((data.length < 1) && (start==0)) return;
        	
        	//If already started then check for at least two
        	if((data.length < 2) && (start==1)) end = 1;
        		
        	if(end == 1) return;
        	
        	if(debug==1) System.out.println("DATA IS: "+data);
        	
        	//Trim desc
        	desc = desc.trim();
        	
        	
        	//Trim and convert data to string whilst formatting it
        	temp = (formatData(data)).trim();
        	String test = temp.toLowerCase();
        	
        	//System.out.println(test);
        	
        	if(temp.length()<1) return;
        	
        	//Has description end yet (start = 1)
        	if(start == 0)
        	{
        		//If temp is longer than description compare 
        		if(temp.length() > desc.length())
        		{
        			
        			//If the start of temp equals desc
        			if (desc.equals((test.substring(0, desc.length())).trim()))
        			{
	        			start = 1;
	        			text = text + temp + " ";
	        			return;
        			}
        		}
        		//If temp is smaller than desc
        		else
        		{
        			
        			//If the start of temp equals desc
        			if (test.equals((desc.substring(0, test.length())).trim()))
        			{
	        			text = text + temp + " ";
	        			//System.out.println("TEMP IS "+temp);
	        			//Remove from desc what has already been stored in text
	        			desc = desc.substring(test.length(), desc.length());
	        			return;
        			}
        		}
        	}
        	// if start then append to text until empty text
        	else
        	{

        		text = text + temp + " ";
        	}
        	
        }
    }

}
